﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections.Specialized;
using System.Text.RegularExpressions;
using System.Web;

namespace xunsearch.lib
{
    /// <summary>
    /// 搜索类, 执行搜索功能
    /// </summary>
    public class XSSearch:XSServer
    {
        /// <summary>
        /// 搜索结果默认分页数量
        /// </summary>
        public const int PAGE_SIZE = 10;

        public const string LOG_DB = "log_db";

        private int _defaultOp = xs_cmd.CMD_QUERY_OP_AND;
        private Dictionary<string, bool> _prefix=new Dictionary<string,bool>();
        private bool _fieldSet;
        private int _lastCount;
        private string _charset,_query;
        private Dictionary<string, Dictionary<string, string>> _facets = new Dictionary<string, Dictionary<string, string>>();
        private int _limit=0, _offset=0;

        private string _lastDb, _curDb = "";
        private List<string> _curDbs, _lastDbs;
        private bool _resetScheme=false;
        private List<string> _terms;
        private int _count = 0;
        private Dictionary<string,object> _highlight;
        private string _highlight_str = "";

        public XSSearch() 
        {

        }
        public XSSearch(string conn, XS xs) : base(conn, xs) 
        {
        }
        /// <summary>
        /// 连接搜索服务端并初始化
        /// 每次重新连接后所有的搜索语句相关设置均被还原
        /// </summary>
        /// <param name="conn"></param>
        public void open(string conn) 
        {
            base.open(conn);
            _prefix = new Dictionary<string, bool>();
            _fieldSet = false;
            _lastCount = 0;
        }

        /// <summary>
        /// 设置字符集
        /// 默认字符集是 UTF-8, 如果您提交的搜索语句和预期得到的搜索结果为其它字符集, 请先设置
        /// </summary>
        /// <param name="charset"></param>
        /// <returns></returns>
        public XSSearch setCharset(string charset) 
        {
            _charset = charset.ToUpper();
            if (_charset == "UTF8") 
            {
                _charset = "UTF-8";
            }
            return this;
        }

        /// <summary>
        /// 开启模糊搜索
        /// 默认情况只返回包含所有搜索词的记录, 通过本方法可以获得更多搜索结果
        /// </summary>
        /// <param name="value">设为 true 表示开启模糊搜索, 设为 false 关闭模糊搜索</param>
        /// <returns></returns>
        public XSSearch setFuzzy(bool value = true)
        {
            _defaultOp = value ? xs_cmd.CMD_QUERY_OP_OR : xs_cmd.CMD_QUERY_OP_AND;
            return this;
        }


        /// <summary>
        /// 设置百分比/权重剔除参数
        /// </summary>
        /// <param name="percent">剔除匹配百分比低于此值的文档, 值范围 0-100</param>
        /// <param name="weight">剔除权重低于此值的文档, 值范围 0.1-25.5, 0 表示不剔除</param>
        /// <returns></returns>
        public XSSearch setCutOff(int percent, int weight = 0) 
        {
            percent = Math.Max(0, Math.Min(100, percent));
            weight = Math.Max(0, weight * 10 & 255);
            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_CUTOFF, percent, weight);
            execCommand(cmd);
            return this;
        }


        /// <summary>
        /// 设置在搜索结果文档中返回匹配词表
        /// </summary>
        /// <param name="value">设为 true 表示开启返回, 设为 false 关闭该功能, 默认是不开启</param>
        /// <returns></returns>
        public XSSearch setRequireMatchedTerm(bool value = true) 
        {
            int arg1 = xs_cmd.CMD_SEARCH_MISC_MATCHED_TERM;
            int arg2 = value ? 1 : 0;
            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_MISC, arg1, arg2);
            execCommand(cmd);
            return this;
        }

        /// <summary>
        /// 开启自动同义词搜索功能
        /// </summary>
        /// <param name="value">设为 true 表示开启同义词功能, 设为 false 关闭同义词功能</param>
        /// <returns></returns>
        public XSSearch setAutoSynonyms(bool value = true) 
        {
            int flag = xs_cmd.CMD_PARSE_FLAG_BOOLEAN | xs_cmd.CMD_PARSE_FLAG_PHRASE | xs_cmd.CMD_PARSE_FLAG_LOVEHATE;
            if (value) 
            {
                flag |= xs_cmd.CMD_PARSE_FLAG_AUTO_MULTIWORD_SYNONYMS;
            }
            XSCommand cmd = new XSCommand(xs_cmd.CMD_QUERY_PARSEFLAG, flag);
            execCommand(cmd);
            return this;
        }

        /// <summary>
        /// 设置同义词搜索的权重比例
        /// </summary>
        /// <param name="value">取值范围 0.01-2.55, 1 表示不调整</param>
        /// <returns></returns>
        public XSSearch setSynonymScale(int value) 
        {
            int arg1 = xs_cmd.CMD_SEARCH_MISC_SYN_SCALE;
            int arg2 = Math.Max(0, value * 100 & 255);
            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_MISC, arg1, arg2);
            execCommand(cmd);
            return this;
        }

        /// <summary>
        /// 获取当前库内的全部同义词列表
        /// </summary>
        /// <param name="limit">数量上限, 若设为 0 则启用默认值 100 个</param>
        /// <param name="offset">偏移量, 即跳过的结果数量, 默认为 0</param>
        /// <param name="stemmed">是否包含处理过的词根同义词, 默认为 false 表示否</param>
        /// <returns>同义词记录数组, 每个词条为键, 同义词条组成的数组为值</returns>
        public Dictionary<string, List<string>> getAllSynonyms(int limit = 0, int offset = 0, bool stemmed = false) 
        {
            string page = "";
            if(limit > 0){
                int i1=offset;
                int i2=offset>>7;
                int i3=offset>>15;
                int i4=offset>>23;
                int i5=limit;
                int i6=limit>>7;
                int i7=limit>>15;
                int i8=limit>>23;
                page = string.Format("{0}{1}{2}{3}{4}{5}{6}{7}", i1, i2, i3, i4, i5, i6, i7, i8);
            }
            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_GET_SYNONYMS, 0, 0, "", page);
            cmd.arg1 = stemmed ? 1 : 0;
            XSCommand res = execCommand(cmd, xs_cmd.CMD_OK_RESULT_SYNONYMS);
            string buf = res.buf;
            Dictionary<string, List<string>> ret = new Dictionary<string, List<string>>();
            if (buf != "") 
            {
                foreach (string line in buf.Split('\n')) 
                {
                    List<string> value = line.Split('\t').ToList();
                    string key = value[0];
                    value.RemoveAt(0);
                    ret.Add(key, value);
                }
            }

            return ret;
        }

        /// <summary>
        /// 获取解析后的搜索语句
        /// </summary>
        /// <param name="query">搜索语句, 若传入 null 使用默认语句</param>
        /// <returns>返回解析后的搜索语句</returns>
        public string getQuery(string query) 
        {
            if (query == null)
            {
                query = "";
            }
            //query = preQueryString(query);
            XSCommand cmd = new XSCommand(xs_cmd.CMD_QUERY_GET_STRING, 0, _defaultOp, query);
            XSCommand res = execCommand(cmd, xs_cmd.CMD_OK_QUERY_STRING);
            //--todo
            if (res.buf.IndexOf("VALUE_RANGE") > 0) 
            {
                string regex = @"/(VALUE_RANGE) (\d+) (\S+) (\S+?)(?=\))/";
                res.buf=Regex.Replace(res.buf, regex, new MatchEvaluator(formatValueRange));
            }
            if (res.buf.IndexOf("VALUE_GE") > 0)
            {
                string regex = @"/(VALUE_[GL]E) (\d+) (\S+?)(?=\))/";
                res.buf = Regex.Replace(res.buf, regex, new MatchEvaluator(formatValueRange));
            }
            return XS.Convert(res.buf,_charset,"UTF-8");
        }

        /// <summary>
        /// 设置默认搜索语句
        /// 用于不带参数的 {@link count} 或 {@link search} 以及 {@link terms} 调用
        /// 可与 {@link addWeight} 组合运用
        /// </summary>
        /// <param name="query">搜索语句, 设为 null 则清空搜索语句, 最大长度为 80 字节</param>
        /// <returns></returns>
        public XSSearch setQuery(string query) 
        {
            query = Tool.ToUTF8(query);
            clearQuery();
            if (query != "") 
            {
                _query = query;
                addQueryString(query);
            }

            return this;
        }


        /// <summary>
        /// 设置多字段组合排序方式
        /// 当您需要根据多个字段的值按不同的方式综合排序时, 请使用这项
        /// </summary>
        /// <param name="fields"></param>
        /// <param name="reverse">是否为倒序显示, 默认为正向, 此处和 {@link setSort} 略有不同</param>
        /// <param name="relevance_first">是否优先相关性排序, 默认为否</param>
        /// <returns></returns>
        public XSSearch setMultiSort(Dictionary<string, bool> fields, bool reverse = false, bool relevance_first = false) 
        {
            // [vno][0/1] (0:reverse,1:asc)
            string buf = "";
            foreach (var de in fields)
            {
                int vno = 0;
                bool asc = false;
                if (de.Value)
                {
                    vno = xs.getField(de.Key, true).vno;
                    asc = de.Value;
                }
                else 
                {
                    vno = xs.getField("false", true).vno;
                    asc = false;
                }
                if (vno != XSFieldScheme.MIXED_VNO) 
                {
                    buf += vno.ToString() + (asc ? 1 : 0).ToString();
                }
            }

            if (buf != "") 
            {
                int type = xs_cmd.CMD_SORT_TYPE_MULTI;
                if (relevance_first)
                    type |= xs_cmd.CMD_SORT_FLAG_RELEVANCE;

                if (!reverse)
                    type |= xs_cmd.CMD_SORT_FLAG_ASCENDING;

                XSCommand cmd = new XSCommand(xs_cmd.CMD_SORT_FLAG_ASCENDING, type, 0, buf);
                execCommand(cmd);
            }

            return this;
        }

        /// <summary>
        /// 设置多字段组合排序方式
        /// 当您需要根据多个字段的值按不同的方式综合排序时, 请使用这项
        /// </summary>
        /// <param name="fields"></param>
        /// <param name="reverse">是否为倒序显示, 默认为正向, 此处和 {@link setSort} 略有不同</param>
        /// <param name="relevance_first">是否优先相关性排序, 默认为否</param>
        /// <returns></returns>
        public XSSearch setMultiSort(string fields, bool reverse = false, bool relevance_first = false) 
        {
            return setSort(fields, !reverse, relevance_first);
        }

        /// <summary>
        /// 设置搜索结果的排序方式
        /// 注意, 每当调用 {@link setDb} 或 {@link addDb} 修改当前数据库时会重置排序设定
        /// 此函数第一参数的用法与 {@link setMultiSort} 兼容, 即也可以用该方法实现多字段排序
        /// </summary>
        /// <param name="field">依据指定字段的值排序, 设为 null 则用默认顺序</param>
        /// <param name="asc">是否为正序排列, 即从小到大, 从少到多, 默认为反序</param>
        /// <param name="relevance_first">是否优先相关性排序, 默认为否</param>
        /// <returns></returns>
        public XSSearch setSort(string field, bool asc = false, bool relevance_first = false) 
        {
            XSCommand cmd;
            if (field == "")
            {
                cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_SORT, xs_cmd.CMD_SORT_TYPE_RELEVANCE);

            }
            else 
            {
                int type = xs_cmd.CMD_SORT_TYPE_VALUE;
                if (relevance_first) 
                {
                    type |= xs_cmd.CMD_SORT_FLAG_RELEVANCE;
                }
                if (asc) 
                {
                    type |= xs_cmd.CMD_SORT_FLAG_ASCENDING;
                }
                int vno = xs.getField(field, true).vno;
                cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_SORT, type, vno);
            }

            execCommand(cmd);
            return this;
        }
        /// <summary>
        /// 设置搜索结果的排序方式
        /// 注意, 每当调用 {@link setDb} 或 {@link addDb} 修改当前数据库时会重置排序设定
        /// 此函数第一参数的用法与 {@link setMultiSort} 兼容, 即也可以用该方法实现多字段排序
        /// </summary>
        /// <param name="field">依据指定字段的值排序, 设为 null 则用默认顺序</param>
        /// <param name="asc">是否为正序排列, 即从小到大, 从少到多, 默认为反序</param>
        /// <param name="relevance_first">是否优先相关性排序, 默认为否</param>
        /// <returns></returns>
        public XSSearch setSort(Dictionary<string, bool> field, bool asc = false, bool relevance_first = false) 
        {
            return setMultiSort(field, asc, relevance_first);
        }

        /// <summary>
        /// 设置结果按索引入库先后排序
        /// 注意, 此项排序不影响相关排序, 权重高的仍会在前面, 主要适合用于布尔检索
        /// </summary>
        /// <param name="asc">是否为正序排列, 即从先到后, 默认为反序</param>
        /// <returns></returns>
        public XSSearch setDocOrder(bool asc = false) 
        {
            int type = xs_cmd.CMD_SORT_TYPE_DOCID | (asc ? xs_cmd.CMD_SORT_FLAG_ASCENDING : 0);
            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_SORT, type);
            execCommand(cmd);
            return this;
        }

        /// <summary>
        /// 设置折叠搜索结果
        /// 注意, 每当调用 {@link setDb} 或 {@link addDb} 修改当前数据库时会重置此项设置
        /// </summary>
        /// <param name="field">依据该字段的值折叠搜索结果, 设为 null 则取消折叠</param>
        /// <param name="num">折叠后只是返最匹配的数据数量, 默认为 1, 最大值 255</param>
        /// <returns></returns>
        public XSSearch setCollapse(string field, int num = 1) 
        {
            int vno = field == "" ? XSFieldScheme.MIXED_VNO : xs.getField(field, true).vno;
            int max = Math.Min(255, num);

            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_COLLAPSE, max, vno);
            execCommand(cmd);

            return this;
        }

        /// <summary>
        /// 添加搜索过滤区间或范围
        /// </summary>
        /// <param name="field"></param>
        /// <param name="from">起始值(不包含), 若设为 null 则相当于匹配 &lt= to (字典顺序)</param>
        /// <param name="to">结束值(包含), 若设为 null 则相当于匹配 >= from (字典顺序)	 </param>
        /// <returns></returns>
        public XSSearch addRange(string field, string from, string to) 
        {
            if (from != "" || to != "")
            {
                if (from.Length > 255 || to.Length > 255) 
                {
                    throw new Exception("Value of range is too long");
                }

                int vno = xs.getField(field).vno;
                from = XS.Convert(from, "UTF-8", _charset);
                to = XS.Convert(to, "UTF-8", _charset);
                XSCommand cmd;
                if (from == "") 
                {
                    cmd = new XSCommand(xs_cmd.CMD_QUERY_VALCMP, xs_cmd.CMD_QUERY_OP_FILTER, vno, to, xs_cmd.CMD_VALCMP_LE.ToString());
                }
                else if (to == "")
                {
                    cmd = new XSCommand(xs_cmd.CMD_QUERY_VALCMP, xs_cmd.CMD_QUERY_OP_FILTER, vno, from, xs_cmd.CMD_VALCMP_GE.ToString());
                }
                else 
                {
                    cmd = new XSCommand(xs_cmd.CMD_QUERY_RANGE, xs_cmd.CMD_QUERY_OP_FILTER, vno, from, to);
                }
                execCommand(cmd);
            }

            return this;
        }


        /// <summary>
        /// 添加权重索引词
        /// 无论是否包含这种词都不影响搜索匹配, 但会参与计算结果权重, 使结果的相关度更高
        /// </summary>
        /// <param name="field">索引词所属的字段</param>
        /// <param name="term">索引词</param>
        /// <param name="weight">权重计算缩放比例</param>
        /// <returns></returns>
        public XSSearch addWeight(string field, string term, int weight = 1) 
        {
            return addQueryTerm(field, term, xs_cmd.CMD_QUERY_OP_AND_MAYBE, weight);
        }

        /// <summary>
        /// 设置分面搜索记数
        /// 用于记录匹配搜索结果中按字段值分组的数量统计, 每次调用 {@link search} 后会还原设置 
        /// 对于多次调用 $exact 参数以最后一次为准, 只支持字段值不超过 255 字节的情况
        /// </summary>
        /// <param name="field">要进行分组统计的字段或字段组成的数组, 最多同时支持 8 个</param>
        /// <param name="exact">是否要求绝对精确搜索, 这会造成较大的系统开销</param>
        /// <returns></returns>
        public XSSearch setFacets(List<string> field, bool exact = false) 
        {
            string buf = "";
            foreach (string name in field) 
            {
                XSFieldMeta ff = xs.getField(name);
                if (ff.type != XSFieldMeta.TYPE_STRNG) //todo
                {                    
                    throw new Exception(string.Format("Field {0} cann't be used for facets search, can only be string type", name));
                }
                buf += ff.vno.ToString();
            }
            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_FACETS, exact ? 1 : 0, 0, buf);
            execCommand(cmd);
            return this;
        }

        /// <summary>
        /// 读取最近一次分面搜索记数
        /// 必须在某一次 {@link search} 之后调用本函数才有意义
        /// </summary>
        /// <param name="field">读取分面记数的字段, 若为 null 则返回全部分面搜索记录</param>
        /// <returns></returns>
        public object getFacets(string field) 
        {//todo
            if (field==null || field == "") 
            {
                return _facets;
            }
            if(_facets.ContainsKey(field)){
                return _facets[field];
            }
            return new List<string>();
        }

        /// <summary>
        /// 设置当前搜索语句的分词复合等级
        /// 复合等级是 scws 分词粒度控制的一个重要参数, 是长词细分处理依据, 默认为 3, 值范围 0~15
        /// 注意: 这个设置仅直对本次搜索有效, 仅对设置之后的 {@link setQuery} 起作用, 由于 query
        /// 设计的方式问题, 目前无法支持搜索语句单字切分, 但您可以在模糊检索时设为 0 来关闭复合分词
        /// </summary>
        /// <param name="level">要设置的分词复合等级</param>
        /// <returns></returns>
        public XSSearch setScwsMulti(int level) 
        {
            if (level >= 0 && level < 16) 
            {
                XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_SCWS_SET, xs_cmd.CMD_SCWS_SET_MULTI, level);
                execCommand(cmd);
            }
            return this;
        }

        /// <summary>
        /// 设置搜索结果的数量和偏移
        /// 用于搜索结果分页, 每次调用 {@link search} 后会还原这2个变量到初始值
        /// </summary>
        /// <param name="limit">数量上限, 若设为 0 则启用默认值 self::PAGE_SIZE</param>
        /// <param name="offset">偏移量, 即跳过的结果数量, 默认为 0</param>
        /// <returns></returns>
        public XSSearch setLimit(int limit, int offset = 0) 
        {
            _limit = limit;
            _offset = offset;
            return this;
        }

        /// <summary>
        /// 设置要搜索的数据库名
        /// 若未设置, 使用默认数据库, 数据库必须位于服务端用户目录下
        /// 对于远程数据库, 请使用 stub 文件来支持
        /// </summary>
        /// <param name="name"></param>
        /// <returns></returns>
        public XSSearch setDb(string name) 
        {
            execCommand(new XSCommand(xs_cmd.CMD_SEARCH_SET_DB, 0, 0, name));
            _lastDb = _curDb;
            _lastDbs = _curDbs;
            _curDb = name;
            _curDbs = new List<string>();
            return this;
        }

        /// <summary>
        /// 添加搜索的数据库名, 支持多库同时搜索
        /// </summary>
        /// <param name="name"></param>
        /// <returns></returns>
        public XSSearch addDb(string name) 
        {
            execCommand(new XSCommand(xs_cmd.CMD_SEARCH_ADD_DB, 0, 0, name));
            _curDbs.Add(name);
            return this;
        }


        /// <summary>
        /// 标记字段方案重置
        /// </summary>
        public void markResetScheme() 
        {
            _resetScheme = true;
        }

        /// <summary>
        /// 获取搜索语句中的高亮词条列表
        /// </summary>
        /// <param name="query">搜索语句, 若传入 null 使用默认语句, 最大长度为 80 字节</param>
        /// <param name="convert">是否进行编码转换, 默认为 true</param>
        /// <returns></returns>
        public List<string> terms(string query="",bool convert=true)
        {
            List<string> ret = new List<string>();
            query=query==""?"":preQueryString(query);
            if (query == "" && _terms != null && _terms.Count > 0)
            {
                ret = _terms;
            }
            else 
            {
                XSCommand cmd = new XSCommand(xs_cmd.CMD_QUERY_GET_TERMS, 0, _defaultOp, query);
                XSCommand res = execCommand(cmd, xs_cmd.CMD_OK_QUERY_TERMS);
                ret = new List<string>();

                string[] tmps = res.buf.Split(' ');
                for (int i = 0; i < tmps.Length; i++) 
                {
                    if (tmps[i] == "" || tmps[i].IndexOf(":") > 0) 
                    {
                        continue;
                    }
                    ret.Add(tmps[i]);
                }

                if (query == "") 
                {
                    _terms = ret;
                }
            }

            return convert ? XS.Convert(ret, _charset, "UTF-8") : ret;
        }


        /// <summary>
        /// 估算搜索语句的匹配数据量
        /// </summary>
        /// <param name="query">搜索语句, 若传入 null 使用默认语句, 调用后会还原默认排序方式
        /// 如果搜索语句和最近一次 {@link search} 的语句一样, 请改用 {@link getLastCount} 以提升效率
        /// 最大长度为 80 字节</param>
        /// <returns>匹配的搜索结果数量, 估算数值</returns>
        public int count(string query) 
        {
            query = query == "" ? "" : preQueryString(query);
            if (query == "" && _count > 0) 
            {
                return _count;
            }

            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_GET_TOTAL, 0, _defaultOp, query);
            XSCommand res = execCommand(cmd, xs_cmd.CMD_OK_SEARCH_TOTAL);
            int ret = int.Parse(res.buf);
            if (query == "") 
            {
                _count = ret;
            }
            return ret;
        }

        /// <summary>
        /// 获取匹配的搜索结果文档
        /// 默认提取最匹配的前 self::PAGE_SIZE 个结果
        /// 如需分页请参见 {@link setLimit} 设置, 每次调用本函数后都会还原 setLimit 的设置
        /// </summary>
        /// <param name="query"></param>
        /// <param name="saveHighlight"></param>
        /// <returns></returns>
        public List<XSDocument> search(string query="", bool saveHighlight = true) 
        {
            if (_curDb != LOG_DB && saveHighlight) 
            {//todo
                _highlight_str = query;
            }
            query = query == "" ? "" : preQueryString(query);
            int p1=_offset;
            int p2=_limit > 0 ? _limit : PAGE_SIZE;

            int a1 = p1;
            int a2 = p1 >> 7;
            int a3 = p1 >> 15;
            int a4 = p1 >> 23;
            int b1 = p2;
            int b2 = p2 >> 7;
            int b3 = p2 >> 15;
            int b4 = p2 >> 23;
            string page = string.Format("{0}{1}{2}{3}{4}{5}{6}{7}", (char)a1, (char)a2, (char)a3, (char)a4, (char)b1, (char)b2, (char)b3, (char)b4);
           
            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_GET_RESULT, 0, _defaultOp, query, page);
            XSCommand res = execCommand(cmd, xs_cmd.CMD_OK_RESULT_BEGIN);
            _lastCount = Tool.ToNum(res.buf);

            //load vno map to name of fields
            List<XSDocument> ret = new List<XSDocument>();
            _facets = new Dictionary<string,Dictionary<string,string>>();
            Dictionary<string, string> vnoes = xs.getScheme().getVnoMap();

            //get result documents
            XSDocument doc=null;
            while (true) 
            {
                res = getRespond();
                if (res.cmd == xs_cmd.CMD_SEARCH_RESULT_FACETS)
                {
                    int off = 0;
                    while ((off + 6) < res.buf.Length)
                    {
                        string temp = res.buf.Substring(off, 6);
                        byte[] buf1 = Tool.ToByteArray(temp);
                        //unpack('Cvno/Cvlen/Inum'
                        string vno = buf1[0].ToString();
                        int vlen = buf1[1];
                        int num = (buf1[2] << 23) + (buf1[3] << 15) + (buf1[4] << 7) + buf1[5];
                        if (vnoes.ContainsKey(vno))
                        {
                            string name = vnoes[vno];
                            string value = res.buf.Substring(off + 6, vlen);
                            if (!_facets.ContainsKey(name))
                            {
                                _facets.Add(name, new Dictionary<string, string>());
                            }
                            _facets[name].Add(value, num.ToString());
                        }
                        off += vlen + 6;
                    }
                }
                else if (res.cmd == xs_cmd.CMD_SEARCH_RESULT_DOC)
                {
                    //got new doc
                    doc = new XSDocument(res.buf, _charset);
                    ret.Add(doc);
                }
                else if (res.cmd == xs_cmd.CMD_SEARCH_RESULT_FIELD)
                {
                    if (doc != null)
                    {                        
                        string name = vnoes.ContainsKey(res.arg.ToString()) ? vnoes[res.arg.ToString()] : res.arg.ToString();
                        doc.setField(name, res.buf);
                    }
                }
                else if (res.cmd == xs_cmd.CMD_SEARCH_RESULT_MATCHED)
                {
                    if (doc != null)
                    {
                        doc.setField("matched", res.buf, true);
                    }
                }
                else if (res.cmd == xs_cmd.CMD_OK && res.arg == xs_cmd.CMD_OK_RESULT_END)
                {
                    // got the end
                    break;
                }
                else 
                {
                    string msg = string.Format("Unexpected respond in search {CMD:{0}, ARG:{1}}", res.cmd, res.arg);
                    throw new Exception(msg);
                }
            }
            foreach (var d in ret)
            {
                string buf = d.__get("subject");
            }
            if (query == "") 
            {
                _count = _lastCount;
                if (_curDb != LOG_DB) 
                {
                    logQuery();
                    if (saveHighlight) 
                    {
                        initHighlight();
                    }
                }
            }
            _limit = _offset = 0;
            return ret;
        }

        /// <summary>
        /// 获取最近那次搜索的匹配总数估值
        /// </summary>
        /// <returns></returns>
        public int getLastCount() 
        {
            return _lastCount;
        }

        /// <summary>
        /// 获取搜索数据库内的数据总量
        /// </summary>
        /// <returns>数据总量</returns>
        public int getDbTotal() 
        {
            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_DB_TOTAL);
            XSCommand res = execCommand(cmd, xs_cmd.CMD_OK_DB_TOTAL);
            return int.Parse(res.buf);
        }

        /// <summary>
        /// 获取热门搜索词列表
        /// </summary>
        /// <param name="limit">需要返回的热门搜索数量上限, 默认为 6, 最大值为 50</param>
        /// <param name="type">排序类型, 默认为 total(搜索总量), 可选值还有 lastnum(上周), currnum(本周)</param>
        /// <returns></returns>
        public Dictionary<string, string> getHotQuery(int limit = 6, string type = "total") 
        {
            Dictionary<string, string> ret = new Dictionary<string, string>();
            limit = Math.Max(1, Math.Min(50, limit));

            //query from log_db
            xs.setScheme((XSFieldScheme)XSFieldScheme.logger());
            try
            {
                setDb(LOG_DB).setLimit(limit);
                if (type != "lastnum" && type != "currnum")
                {
                    type = "total";
                }

                List<XSDocument> result = search(type + ":1");
                foreach (XSDocument doc in result)
                {
                    string body = doc.__get("body");
                    ret.Add(body, doc.f(type));
                }
                restoreDb();
            }
            catch (Exception ex)
            {

            }

            xs.restoreScheme();
            return ret;
        }

        /// <summary>
        /// 获取相关搜索词列表
        /// </summary>
        /// <param name="query"></param>
        /// <param name="limit"></param>
        /// <returns></returns>
        public List<string> getRelatedQuery(string query = "", int limit = 6) 
        {
            List<string> ret = new List<string>();
            limit = Math.Max(1, Math.Min(20, limit));

            //Simple to disable query with field filter
            if (query == "") 
            {
                query = cleanFieldQuery(_query);
            }

            if (query == "" || query.IndexOf(":") < 0) 
            {
                return ret;
            }

            //Search the log database
            int op = _defaultOp;

            xs.setScheme((XSFieldScheme)XSFieldScheme.logger());
            try
            {
                List<XSDocument> result = setDb(LOG_DB).setFuzzy().setLimit(limit + 1).search(query);
                foreach (XSDocument doc in result)
                {
                    doc.setCharset(_charset);
                    string body = doc.__get("body");
                    if (body.IndexOf(query) < 0)
                    {
                        continue;
                    }
                    ret.Add(body);
                    if (ret.Count == limit)
                    {
                        break;
                    }
                }
            }
            catch (Exception ex) 
            {
            }

            restoreDb();
            xs.restoreScheme();
            _defaultOp = op;


            return ret;
        }

        /// <summary>
        /// 获取展开的搜索词列表
        /// </summary>
        /// <param name="query"></param>
        /// <param name="limit"></param>
        /// <returns></returns>
        public List<string> getExpandedQuery(string query = "", int limit = 10) 
        {
            List<string> ret = new List<string>();
            limit = Math.Max(1, Math.Min(20, limit));

            try
            {
                string buf = XS.Convert(query, "UTF-8", _charset);
                XSCommand cmd = new XSCommand(xs_cmd.CMD_QUERY_GET_EXPANDED, 0, limit, buf);
                XSCommand res = execCommand(cmd, xs_cmd.CMD_OK_RESULT_BEGIN);

                //
                while (true)
                {
                    res = getRespond();
                    if (res.cmd == xs_cmd.CMD_SEARCH_RESULT_FIELD)
                    {
                        ret.Add(XS.Convert(res.buf, _charset, "UTF-8"));
                    }
                    else if (res.cmd == xs_cmd.CMD_OK && res.arg == xs_cmd.CMD_OK_RESULT_END)
                    {
                        break;
                    }
                    else
                    {
                        string msg = string.Format("Unexpected respond in search {CMD:{0}, ARG:{1}}", res.cmd, res.arg);
                        throw new Exception(msg);
                    }
                }
            }
            catch (Exception ex) 
            {

            }

            return ret;
        }

        /// <summary>
        /// 获取修正后的搜索词列表
        /// 通常当某次检索结果数量偏少时, 可以用该函数设计 "你是不是要找: ..." 功能
        /// </summary>
        /// <param name="query">需要展开的前缀, 可为拼音、英文、中文</param>
        /// <returns></returns>
        public List<string> getCorrectedQuery(string query = "") 
        {
            List<string> ret = new List<string>();

            try
            {
                if (query == "") 
                {
                    if (_count > 0 && _count > Math.Ceiling(getDbTotal() * 0.001)) 
                    {
                        return ret;
                    }
                    query = cleanFieldQuery(_query);
                }

                if (query == "" || query.IndexOf(":") >0) 
                {
                    return ret;
                }
                string buf = XS.Convert(query, "UTF-8", _charset);
                XSCommand cmd = new XSCommand(xs_cmd.CMD_QUERY_GET_CORRECTED, 0, 0, buf);
                XSCommand res = execCommand(cmd, xs_cmd.CMD_OK_QUERY_CORRECTED);
                if (res.buf != "") 
                {
                    ret = XS.Convert(res.buf,_charset,"UTF-8").Split('\n').ToList();
                }
            }
            catch (Exception ex) 
            {
            }

            return ret;
        }

        /// <summary>
        /// 添加搜索日志关键词到缓冲区里
        /// 需要调用 {@link XSIndex::flushLogging} 才能确保立即刷新, 否则要隔一段时间
        /// </summary>
        /// <param name="query">需要记录的数据</param>
        /// <param name="wdf">需要记录的次数, 默认为 1</param>
        public void addSearchLog(string query, int wdf = 1) 
        {
            XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_ADD_LOG, 0, 0, query);
            if (wdf > 1) 
            {
                cmd.buf1 = wdf.ToString();
            }
            execCommand(cmd, xs_cmd.CMD_OK_LOGGED);
        }

        /// <summary>
        /// 搜索结果字符串高亮处理
        /// 对搜索结果文档的字段进行高亮、飘红处理, 高亮部分加上 em 标记
        /// </summary>
        /// <param name="value"></param>
        /// <param name="strtr"></param>
        /// <returns></returns>
        public string highlight(string value, bool strtr = false) 
        {
            if (string.IsNullOrWhiteSpace(value)) 
            {
                return "";
            }
            if (_highlight_str!="" && _highlight!=null && _highlight.Count<0) 
            {
                initHighlight();
            }
            if (_highlight.ContainsKey("pattern")) 
            {//todo
                value = value.Replace(_highlight["pattern"].ToString(), _highlight["replace"].ToString());
            }
            if (_highlight.ContainsKey("pairs")) 
            {
                value = strtr ? value + _highlight["pairs"] : value.Replace("pairs", _highlight["pairs"].ToString());
            }

            return value;
        }

        /// <summary>
        /// 记录搜索语句
        /// 主要是用于相关搜索, 修正搜索等功能, 为避免记录一些杂乱无用的搜索信息,
        /// 系统会先检测这条语句是否符合记录需求, 力争记录一些规范清洁的数据
        /// </summary>
        /// <param name="query"></param>
        public void logQuery(string query = "") 
        {
            List<string> __terms = new List<string>();
            if (isRobotAgent()) 
            {
                return;
            }
            if (query != "")
            {
                __terms = terms(query, false);
            }
            else 
            {
                query = _query;
                if (_lastCount < 0 || (_defaultOp == xs_cmd.CMD_QUERY_OP_OR && query.IndexOf(" ") > 0) ||
                    query.IndexOf(" OR ")>0 || query.IndexOf(" NOT ")>0 || query.IndexOf(" XOR ")>0) 
                {
                    return;
                }
                __terms = terms("", false);
            }

            string log = "";
            int pos =0, max = 0;
            foreach (string term in __terms) 
            {
                int pos1 = (pos > 3 && term.Length == 6) ? pos - 3 : pos;
                int pos2 = query.IndexOf(term, pos1);
                if (pos2 < 0) 
                {
                    continue;
                }
                if (pos2 == pos) 
                {
                    log += term;
                }
                else if (pos2 < pos)
                {
                    log += term.Substring(0, 3);
                }
                else 
                {
                    if (++max > 3 || log.Length > 42) 
                    {
                        break;
                    }
                    log += " " + term;
                }
                pos = pos2 + term.Length;
            }
            // run the command, filter for single word character
            log = log.Trim();
            if (log.Length < 2 || log.Length == 3 && (int)System.Text.ASCIIEncoding.ASCII.GetBytes(log)[0] > 0x80) 
            {
                return;
            }

            addSearchLog(log);
        }

        /// <summary>
        /// 清空默认搜索语句
        /// </summary>
        public void clearQuery() 
        {
            XSCommand cmd = new XSCommand(xs_cmd.CMD_QUERY_INIT);
            if (_resetScheme) 
            {
                cmd.arg1 = 1;
                _prefix = new Dictionary<string, bool>();
                _fieldSet = false;
                _resetScheme = false;
            }

            execCommand(cmd);
            _query = "";
            _count = 0;
            _terms = null;
        }

        /// <summary>
        /// 增加默认搜索语句
        /// </summary>
        /// <param name="query">搜索语句</param>
        /// <param name="addOp">与旧语句的结合操作符, 如果无旧语句或为空则这此无意义, 支持的操作符有:</param>
        /// <param name="scale">scale 权重计算缩放比例, 默认为 1表示不缩放, 其它值范围 </param>
        /// <returns></returns>
        public string addQueryString(string query, int addOp = xs_cmd.CMD_QUERY_OP_AND, int scale = 1) 
        {
            query = preQueryString(query);
            string bscale = (scale > 1) ? string.Format("{0}", scale * 100) : "";
            XSCommand cmd = new XSCommand(xs_cmd.CMD_QUERY_PARSE, addOp, _defaultOp, query, bscale);
            execCommand(cmd);
            return query;
        }

        /// <summary>
        /// 增加默认搜索词汇
        /// </summary>
        /// <param name="field"></param>
        /// <param name="term"></param>
        /// <param name="addOp"></param>
        /// <param name="scale"></param>
        /// <returns></returns>
        public XSSearch addQueryTerm(string field, string term, int addOp = xs_cmd.CMD_QUERY_OP_AND, int scale = 1) 
        {
            term = term.ToLower();
            term = XS.Convert(term, "UTF-8", _charset);
            string bscale = (scale > 0 && scale != 1) ? string.Format("{0}", scale * 100) : "";
            int vno = field == "" ? XSFieldScheme.MIXED_VNO : xs.getField(field, true).vno;


            XSCommand cmd = new XSCommand(xs_cmd.CMD_QUERY_TERM, addOp, vno, term, bscale);
            execCommand(cmd);
            return this;
        }

        /// <summary>
        /// 还原搜索 DB
        /// </summary>
        private void restoreDb() 
        {
            string db = _lastDb;
            List<string> dbs = _lastDbs;
            setDb(db);
            foreach (string name in dbs) 
            {
                addDb(name);
            }
        }

        /// <summary>
        /// 搜索语句的准备工作
        /// 登记相关的字段前缀并给非布尔字段补上括号, 首次搜索必须通知服务端关于 cutlen, numeric 字段的设置
        /// </summary>
        /// <param name="query"></param>
        /// <returns></returns>
        private string preQueryString(string query) 
        {
            query = query.Trim();
            if (_resetScheme == true) 
            {
                clearQuery();
            }
            // init special field here
            initSpecialField();
            string newQuery = "";
            List<string> parts = Tool.preg_split("/[ \t\r\n]+/", query).ToList();
            foreach (string part in parts)
            {
                if (part == "") 
                {
                    continue;
                }
                if (newQuery != "") 
                {
                    newQuery += " ";
                }
                int pos = part.IndexOf(":", 1);
                int i=0;
                if (pos > 0) 
                {

                    for (i = 0; i < pos; i++) 
                    {
                        if ("+-~(".IndexOf(part[i]) < 0) 
                        {
                            break;
                        }
                    }
                    string name = part.Substring(i, pos - i);
                    XSFieldMeta field = xs.getField(name, false);
                    if (field != null && field.vno != XSFieldScheme.MIXED_VNO) 
                    {
                        regQueryPrefix(name);
                        if (field.hasCustomTokenizer()) 
                        {
                            string prefix = i > 0 ? part.Substring(0, i) : "";
                            string suffix = "";

                            string value = part.Substring(0, pos + 1);
                            if (value.Substring(value.Length-1, 1) == ")") 
                            {
                                suffix = ")";
                                value = value.Substring(0, value.Length - 1);
                            }

                            List<string> _terms=new List<string>();
                            XSTokenizer xt = (XSTokenizer)field.getCustomTokenizer();
                            List<string> tokens = xt.getTokens(name);
                            foreach (string term in tokens) 
                            {
                                if(!_terms.Contains(term.ToLower()))
                                {
                                    _terms.Add(term.ToLower());
                                }
                            }
                            newQuery = prefix + name + ":" + string.Join(" " + name + ":", _terms.ToArray()) + suffix;
                        }
                        else if (part.Substring(pos + 1, 1) != "(" && Tool.CheckEncode(part))
                        {
                            newQuery += part.Substring(0, pos + 1) + "(" + part.Substring(pos + 1) + ")";
                        }
                        else 
                        {
                            newQuery += part;
                        }

                        continue;
                    }
                }
                if (part.Length > 1 && (part[0] == '+' || part[0] == '-') &&
                    part[1] != '(' && Tool.CheckEncode(part)) 
                {
                    newQuery += part.Substring(0, 1) + "(" + part.Substring(1) + ")";
                    continue;
                }
                newQuery += part;
            }
            return XS.Convert(newQuery, "UTF-8", _charset);
        }

        /// <summary>
        /// 登记搜索语句中的字段
        /// </summary>
        /// <param name="name">字段名称</param>
        private void regQueryPrefix(string name) 
        {
            XSFieldMeta field= xs.getField(name,false);
            if (!_prefix.ContainsKey(name) &&
                field!=null && field.vno!=XSFieldScheme.MIXED_VNO
                )                 
            {
                int type = field.isBoolIndex() ? xs_cmd.CMD_PREFIX_BOOLEAN : xs_cmd.CMD_PREFIX_NORMAL;
                XSCommand cmd = new XSCommand(xs_cmd.CMD_QUERY_PREFIX, type, field.vno, name);
                execCommand(cmd);
                _prefix.Add(name, true);
            }
        }

        /// <summary>
        /// 设置字符型字段及裁剪长度
        /// </summary>
        private void initSpecialField() 
        {
            if (_fieldSet)
                return;
            foreach (var de in xs.getAllFields())
            {
                if (de.Value.cutlen != 0)
                {
                    decimal num = de.Value.cutlen / 10;
                    ;
                    int len = Math.Min(127, (int)Math.Ceiling(num));
                    XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_CUT, len, de.Value.vno);
                    execCommand(cmd);
                }
                if (de.Value.isNumeric()) 
                {
                    XSCommand cmd = new XSCommand(xs_cmd.CMD_SEARCH_SET_NUMERIC, 0, de.Value.vno);
                    execCommand(cmd);
                }
            }
            _fieldSet = true;
        }

        /// <summary>
        /// 清除查询语句中的字段名、布尔字段条件
        /// </summary>
        /// <param name="query">查询语句</param>
        /// <returns></returns>
        private string cleanFieldQuery(string query) 
        {
            query = query.Replace(" AND ", " ").Replace(" OR ", " ");
            if (query.IndexOf(":") > 0) 
            {
                string regex = @"/(^|\s)([0-9A-Za-z_\.-]+):([^\s]+)/";
                return Regex.Replace(query, regex, new MatchEvaluator(cleanFieldCallback));
            }
            return query;
        }

        /// <summary>
        /// 清除布尔字段查询语句和非布尔的字段名
        /// 用于正则替换回调函数, 净化 {@link getCorrectedQuery} 和 {@link getRelatedQuery} 中的搜索语句
        /// </summary>
        /// <param name="match">正则匹配的部分, [1]:prefix [2]:field, [3]:data</param>
        /// <returns></returns>
        private string cleanFieldCallback(Match match)
        {
            XSFieldMeta _field = xs.getField(match.Value[2].ToString(), false);
            if (_field == null) 
            {
                return match.Value[0].ToString();
            }
            if (_field.isBoolIndex()) 
            {
                return "";
            }
            string m3 = match.Value[3].ToString();
            if (m3.Substring(0, 1) == "(" && m3.Substring(m3.Length - 1, 1) == ")") 
            {
                m3 = m3.Substring(1, m3.Length - 1);
            }
            return match.Value[1] + m3;

        }

        /// <summary>
        /// 初始始化高亮替换数据
        /// </summary>
        private void initHighlight() 
        {
            List<string> _terms = new List<string>();
            List<string> tmps = terms(_highlight_str, false);//todo
            int i=0,j=0,k=0;
            for (i = 0; i < tmps.Count; i++)
            {
                if (tmps[i].Length != 6 || (int)tmps[i].Substring(0, 1)[0] < 0xc0) 
                {
                    _terms.Add(XS.Convert(tmps[i], _charset, "UTF-8"));
                    continue;
                }

                // auto fixed duality in libscws
                // ABC => AB,BC => ABC,BC,AB
                // ABCD => AB,BC,CD => CD,ABC,BC,AB
                // ABCDE => AB,BC,CD,DE => CDE,DE,CD,ABC,BC,AB
                for (j = i + 1; j < tmps.Count; j++) 
                {
                    if (tmps[j].Length != 6 || tmps[j].Substring(0, 3) != tmps[j - 1].Substring(3, 3)) {
                        break;
                    }
                }
                if ((k = (j - i)) == 1)
                {
                    _terms.Add(XS.Convert(tmps[i], _charset, "UTF-8"));
                }
                else 
                {
                    i = j - 1;
                    while ((k--)>0) 
                    {
                        j--;
                        if ((k & 1)>0)
                        {
                            _terms.Add(XS.Convert(tmps[j - 1].Substring(0, 3) + tmps[j], _charset, "UTF-8"));
                        }
                        _terms.Add(XS.Convert(tmps[j], _charset, "UFT-8"));
                    }
                }                
            }
            Dictionary<string, string> pairs = new Dictionary<string, string>();
            List<string> pattern = new List<string>();
            List<string> replace = new List<string>();
            foreach (string term in _terms) 
            {
                if (!Regex.IsMatch(term, "^[a-zA-Z]"))
                {
                    pairs.Add(term, "<em>" + term + "</em>");
                }
                else 
                {//todo
                    pattern.Add(term.Replace("+", "\\+").Replace("/", "\\/")+"/i");
                    replace.Add("<em>$0{/em}");
                }
            }
            _highlight = new Dictionary<string, object>();
            if (pairs.Count > 0) 
            {
                _highlight.Add("pairs", pairs);
            }
            if (pattern.Count > 0) 
            {
                _highlight.Add("pattern", pattern);
                _highlight.Add("replace", replace);
            }
        }

        /// <summary>
        /// Format the value range/ge
        /// </summary>
        /// <param name="match"></param>
        /// <returns></returns>
        private string formatValueRange(Match match) 
        {
            XSFieldMeta field = xs.getField(match.Value[2].ToString(), false);
            if (field == null) 
            {
                return match.Value[0].ToString();
            }

            string val1 = "~", val2 = "~";
            if (match.Value.Count() > 5)
            {
                val2 = field.isNumeric() ? xapianUnserialise(match.Value[4].ToString()).ToString() : match.Value[4].ToString();
            }

            if (match.Value[1].ToString() == "VALUE_LE")
            {
                val2 = field.isNumeric() ? xapianUnserialise(match.Value[3].ToString()).ToString() : match.Value[3].ToString();
            }
            else 
            {
                val1 = field.isNumeric() ? xapianUnserialise(match.Value[3].ToString()).ToString() : match.Value[3].ToString();
            }

            return field.name + ":[" + val1 + "," + val2 + "]";
        }

        /// <summary>
        /// Convert a string encoded by xapian to a floating point number
        /// </summary>
        /// <param name="value"></param>
        /// <returns></returns>
        private double xapianUnserialise(string value) 
        {
            if (value == "\x80") 
            {
                return 0.0;
            }
            if (value == Tool.str_repeat("\xff", 9)) 
            {
                return double.MaxValue;
            }
            if (value == "") 
            {
                return double.MinValue;
            }
            int i = 0;
            int c = (int)value[0];
            c ^= (c & 0xc0) >> 1;
            int negative = (c & 0x40)==0 ? 1 : 0;
            int exponent_negative = (c & 0x20) != 0 ? 1 : 0;
            int explen = (c & 0x20)==0 ? 1 : 0;
            int exponent = c & 0x1f;
            if (exponent == 0)
            {
                exponent >>= 2;
                if ((negative ^ exponent_negative) != 0)
                {
                    exponent ^= 0x07;
                }
            }
            else 
            {
                c = (int)value[++i];
                exponent <<= 6;
                exponent |= (c >> 2);
                if ((negative ^ exponent_negative) != 0) 
                {
                    exponent &= 0x07ff;
                }
            }

            int word1 = (c & 0x03) << 24;
            word1 |= ((int)value[++i]) << 16;
            word1 |= ((int)value[++i]) << 8;
            word1 |= ((int)value[++i]);

            int word2 = 0;
            if (negative == 0)
            {
                word1 |= 1 << 26;
            }
            else 
            {
                word1 = 0 - word1;
                if (word2 != 0) 
                {
                    ++word1;
                }
                word2 = 0 - word2;
                word1 &= 0x03ffffff;
            }

            int mantissa = 0;
            if (word2 > 0) 
            {
                mantissa = (int)(word2 / 4294967296.0);
            }
            mantissa += word1;
            mantissa /= 1 << (negative == 1 ? 26 : 27);
            if (exponent_negative > 0) 
            {
                mantissa = 0 - mantissa;
            }
            return Math.Round(mantissa * Math.Pow(2.0, exponent), 2);
        }

        /// <summary>
        /// 
        /// </summary>
        /// <returns>boolean whether the user agent is a robot or search engine</returns>
        private bool isRobotAgent() 
        {
            return false;
        }
    }
}
